{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From D:\\Miniconda\\Miniconda1\\lib\\site-packages\\tensorflow\\python\\util\\tf_should_use.py:170: initialize_all_variables (from tensorflow.python.ops.variables) is deprecated and will be removed after 2017-03-02.\n",
      "Instructions for updating:\n",
      "Use `tf.global_variables_initializer` instead.\n",
      "5000\n",
      "Loss: 1.85245\n",
      "Epsilon: 0.951717130256\n",
      "10000\n",
      "Loss: 274.609\n",
      "Epsilon: 0.905789043856\n",
      "15000\n",
      "Loss: 0.000187911\n",
      "Epsilon: 0.862100896661\n",
      "20000\n",
      "Loss: 0.00012883\n",
      "Epsilon: 0.820543445547\n",
      "25000\n",
      "Loss: 9.64627e-07\n",
      "Epsilon: 0.781012775241\n",
      "30000\n",
      "Loss: 1.82659e-06\n",
      "Epsilon: 0.743410038475\n",
      "35000\n",
      "Loss: 5.24735e-08\n",
      "Epsilon: 0.707641208822\n",
      "40000\n",
      "Loss: 1.44645e-07\n",
      "Epsilon: 0.673616845575\n",
      "45000\n",
      "Loss: 1.87307e-08\n",
      "Epsilon: 0.641251870106\n",
      "50000\n",
      "Loss: 4.91855e-10\n",
      "Epsilon: 0.610465353116\n",
      "55000\n",
      "Loss: 4.13456e-10\n",
      "Epsilon: 0.581180312277\n",
      "60000\n",
      "Loss: 2.84126e-09\n",
      "Epsilon: 0.553323519733\n",
      "65000\n",
      "Loss: 1.36224e-09\n",
      "Epsilon: 0.526825318993\n",
      "65000\n",
      "Loss: 1.15415e-09\n",
      "Epsilon: 0.526825318993\n",
      "70000\n",
      "Loss: 2.00089e-10\n",
      "Epsilon: 0.501619450753\n",
      "75000\n",
      "Loss: 1.49885e-10\n",
      "Epsilon: 0.477642887214\n",
      "80000\n",
      "Loss: 4.12547e-10\n",
      "Epsilon: 0.454835674476\n",
      "85000\n",
      "Loss: 8.29459e-10\n",
      "Epsilon: 0.433140782629\n",
      "85000\n",
      "Loss: 5.26234e-10\n",
      "Epsilon: 0.433140782629\n",
      "90000\n",
      "Loss: 2.33013e-10\n",
      "Epsilon: 0.412503963143\n",
      "95000\n",
      "Loss: 4.06908e-10\n",
      "Epsilon: 0.39287361322\n",
      "100000\n",
      "Loss: 1.04483e-09\n",
      "Epsilon: 0.37420064676\n",
      "105000\n",
      "Loss: 1.6953e-10\n",
      "Epsilon: 0.35643837162\n",
      "110000\n",
      "Loss: 3.59432e-10\n",
      "Epsilon: 0.339542372861\n",
      "115000\n",
      "Loss: 3.66163e-10\n",
      "Epsilon: 0.323470401685\n",
      "115000\n",
      "Loss: 7.85803e-11\n",
      "Epsilon: 0.323470401685\n",
      "120000\n",
      "Loss: 1.1074e-09\n",
      "Epsilon: 0.308182269793\n",
      "125000\n",
      "Loss: 4.11092e-10\n",
      "Epsilon: 0.293639748892\n",
      "130000\n",
      "Loss: 3.64525e-10\n",
      "Epsilon: 0.279806475104\n",
      "135000\n",
      "Loss: 1.44792e-10\n",
      "Epsilon: 0.266647858039\n",
      "135000\n",
      "Loss: 1.61708e-10\n",
      "Epsilon: 0.266647858039\n",
      "140000\n",
      "Loss: 4.56203e-10\n",
      "Epsilon: 0.254130994302\n",
      "140000\n",
      "Loss: 3.47791e-10\n",
      "Epsilon: 0.254130994302\n",
      "145000\n",
      "Loss: 1.68075e-10\n",
      "Epsilon: 0.242224585213\n",
      "150000\n",
      "Loss: 1.34605e-10\n",
      "Epsilon: 0.230898858547\n",
      "155000\n",
      "Loss: 1.01863e-10\n",
      "Epsilon: 0.220125494088\n",
      "160000\n",
      "Loss: 1.36788e-10\n",
      "Epsilon: 0.209877552815\n",
      "165000\n",
      "Loss: 6.2355e-10\n",
      "Epsilon: 0.200129409535\n",
      "165000\n",
      "Loss: 8.97126e-10\n",
      "Epsilon: 0.200129409535\n",
      "170000\n",
      "Loss: 3.4488e-10\n",
      "Epsilon: 0.190856688812\n",
      "175000\n",
      "Loss: 2.02999e-10\n",
      "Epsilon: 0.182036204016\n",
      "180000\n",
      "Loss: 7.6252e-10\n",
      "Epsilon: 0.173645899339\n",
      "185000\n",
      "Loss: 2.14641e-10\n",
      "Epsilon: 0.16566479465\n",
      "190000\n",
      "Loss: 2.51021e-10\n",
      "Epsilon: 0.15807293303\n",
      "195000\n",
      "Loss: 2.59752e-10\n",
      "Epsilon: 0.150851330871\n",
      "200000\n",
      "Loss: 2.68483e-10\n",
      "Epsilon: 0.143981930404\n",
      "205000\n",
      "Loss: 1.06229e-10\n",
      "Epsilon: 0.137447554552\n",
      "210000\n",
      "Loss: 1.47702e-10\n",
      "Epsilon: 0.13123186397\n",
      "215000\n",
      "Loss: 7.13044e-11\n",
      "Epsilon: 0.125319316196\n",
      "220000\n",
      "Loss: 2.452e-10\n",
      "Epsilon: 0.119695126779\n",
      "225000\n",
      "Loss: 1.96451e-10\n",
      "Epsilon: 0.114345232316\n",
      "230000\n",
      "Loss: 8.87667e-11\n",
      "Epsilon: 0.109256255286\n",
      "235000\n",
      "Loss: 2.61207e-10\n",
      "Epsilon: 0.104415470593\n",
      "240000\n",
      "Loss: 2.54659e-10\n",
      "Epsilon: 0.0998107737565\n",
      "245000\n",
      "Loss: 1.02591e-10\n",
      "Epsilon: 0.0954306506344\n",
      "250000\n",
      "Loss: 1.70439e-10\n",
      "Epsilon: 0.0912641486377\n",
      "255000\n",
      "Loss: 2.42471e-10\n",
      "Epsilon: 0.0873008493411\n",
      "260000\n",
      "Loss: 9.24047e-10\n",
      "Epsilon: 0.0835308424322\n",
      "265000\n",
      "Loss: 2.53931e-10\n",
      "Epsilon: 0.0799447009298\n",
      "270000\n",
      "Loss: 1.47884e-10\n",
      "Epsilon: 0.0765334576124\n",
      "275000\n",
      "Loss: 3.78532e-10\n",
      "Epsilon: 0.0732885825946\n",
      "280000\n",
      "Loss: 1.76078e-10\n",
      "Epsilon: 0.070201961999\n",
      "285000\n",
      "Loss: 1.89902e-10\n",
      "Epsilon: 0.0672658776661\n",
      "290000\n",
      "Loss: 7.49424e-11\n",
      "Epsilon: 0.0644729878558\n",
      "295000\n",
      "Loss: 1.22964e-10\n",
      "Epsilon: 0.0618163088889\n",
      "300000\n",
      "Loss: 2.28465e-10\n",
      "Epsilon: 0.0592891976842\n",
      "305000\n",
      "Loss: 2.69938e-10\n",
      "Epsilon: 0.0568853351472\n",
      "310000\n",
      "Loss: 2.15368e-10\n",
      "Epsilon: 0.0545987103696\n",
      "315000\n",
      "Loss: 1.24419e-10\n",
      "Epsilon: 0.0524236055984\n",
      "320000\n",
      "Loss: 2.38651e-10\n",
      "Epsilon: 0.0503545819386\n",
      "325000\n",
      "Loss: 1.16415e-10\n",
      "Epsilon: 0.0483864657534\n",
      "330000\n",
      "Loss: 9.74978e-11\n",
      "Epsilon: 0.0465143357272\n",
      "335000\n",
      "Loss: 1.49885e-10\n",
      "Epsilon: 0.0447335105598\n",
      "340000\n",
      "Loss: 6.40284e-11\n",
      "Epsilon: 0.0430395372607\n",
      "345000\n",
      "Loss: 7.71252e-11\n",
      "Epsilon: 0.0414281800143\n",
      "350000\n",
      "Loss: 1.27329e-10\n",
      "Epsilon: 0.0398954095881\n",
      "355000\n",
      "Loss: 2.07547e-10\n",
      "Epsilon: 0.0384373932577\n",
      "360000\n",
      "Loss: 8.22183e-11\n",
      "Epsilon: 0.0370504852228\n",
      "365000\n",
      "Loss: 6.25732e-11\n",
      "Epsilon: 0.035731217491\n",
      "370000\n",
      "Loss: 1.6953e-10\n",
      "Epsilon: 0.0344762912056\n",
      "375000\n",
      "Loss: 1.9063e-10\n",
      "Epsilon: 0.0332825683974\n",
      "380000\n",
      "Loss: 4.07454e-11\n",
      "Epsilon: 0.0321470641376\n",
      "385000\n",
      "Loss: 4.39468e-10\n",
      "Epsilon: 0.031066939074\n",
      "390000\n",
      "Loss: 3.36149e-10\n",
      "Epsilon: 0.0300394923313\n",
      "395000\n",
      "Loss: 2.13367e-10\n",
      "Epsilon: 0.0290621547576\n",
      "395000\n",
      "Loss: 1.58616e-10\n",
      "Epsilon: 0.0290621547576\n",
      "400000\n",
      "Loss: 1.43882e-10\n",
      "Epsilon: 0.0281324824998\n",
      "405000\n",
      "Loss: 4.7512e-10\n",
      "Epsilon: 0.0272481508931\n",
      "410000\n",
      "Loss: 1.9354e-10\n",
      "Epsilon: 0.0264069486477\n",
      "415000\n",
      "Loss: 3.10683e-10\n",
      "Epsilon: 0.02560677232\n",
      "420000\n",
      "Loss: 5.83532e-10\n",
      "Epsilon: 0.0248456210523\n",
      "425000\n",
      "Loss: 1.59343e-10\n",
      "Epsilon: 0.0241215915699\n",
      "430000\n",
      "Loss: 3.76167e-10\n",
      "Epsilon: 0.0234328734221\n",
      "435000\n",
      "Loss: 5.74801e-11\n",
      "Epsilon: 0.0227777444547\n",
      "440000\n",
      "Loss: 3.92902e-11\n",
      "Epsilon: 0.022154566504\n",
      "445000\n",
      "Loss: 2.96131e-10\n",
      "Epsilon: 0.0215617813007\n",
      "450000\n",
      "Loss: 1.27329e-10\n",
      "Epsilon: 0.0209979065729\n",
      "455000\n",
      "Loss: 5.16593e-11\n",
      "Epsilon: 0.02046153234\n",
      "460000\n",
      "Loss: 2.37378e-10\n",
      "Epsilon: 0.0199513173872\n",
      "465000\n",
      "Loss: 1.15688e-10\n",
      "Epsilon: 0.0194659859112\n",
      "470000\n",
      "Loss: 6.40284e-11\n",
      "Epsilon: 0.0190043243307\n",
      "475000\n",
      "Loss: 6.57747e-10\n",
      "Epsilon: 0.0185651782511\n",
      "480000\n",
      "Loss: 2.27737e-10\n",
      "Epsilon: 0.0181474495785\n",
      "485000\n",
      "Loss: 6.4756e-11\n",
      "Epsilon: 0.0177500937737\n",
      "490000\n",
      "Loss: 3.43789e-11\n",
      "Epsilon: 0.0173721172402\n",
      "495000\n",
      "Loss: 1.92813e-10\n",
      "Epsilon: 0.0170125748398\n",
      "500000\n",
      "Loss: 1.29512e-10\n",
      "Epsilon: 0.0166705675291\n",
      "****************** Agent is in the State 0 **********************\n",
      "Step is  1 ****************** Agent is in the State 10 **********************\n",
      "Step is  2 ****************** Agent is in the State 11 **********************\n",
      "Step is  3 ****************** Agent is in the State 21 **********************\n",
      "Step is  4 ****************** Agent is in the State 31 **********************\n",
      "Step is  5 ****************** Agent is in the State 41 **********************\n",
      "Step is  6 ****************** Agent is in the State 42 **********************\n",
      "Step is  7 ****************** Agent is in the State 43 **********************\n",
      "Step is  8 ****************** Agent is in the State 53 **********************\n",
      "Step is  9 ****************** Agent is in the State 54 **********************\n",
      "Step is  10 ****************** Agent is in the State 64 **********************\n",
      "Step is  11 ****************** Agent is in the State 74 **********************\n",
      "Step is  12 ****************** Agent is in the State 75 **********************\n",
      "Step is  13 ****************** Agent is in the State 76 **********************\n",
      "Step is  14 ****************** Agent is in the State 86 **********************\n",
      "Step is  15 ****************** Agent is in the State 87 **********************\n",
      "Step is  16 ****************** Agent is in the State 97 **********************\n",
      "Step is  17 ****************** Agent is in the State 98 **********************\n",
      "Step is  18 ****************** Agent is in the State 99 **********************\n"
     ]
    }
   ],
   "source": [
    "%matplotlib qt5\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from collections import deque\n",
    "import random\n",
    "import matplotlib.pyplot as plt\n",
    "class Q_Network:\n",
    "    N_Mesh=10\n",
    "    State_Number=N_Mesh*N_Mesh;\n",
    "    Action_Number=4;\n",
    "    Action_List=np.identity(Action_Number)\n",
    "    State_List=np.identity(State_Number)\n",
    "    Epsilon=0\n",
    "    Epsilon_Begin=1\n",
    "    Epsilon_Final=0.01\n",
    "    decay_rate=0.00001\n",
    "    Step=0\n",
    "    Batch_Number=20\n",
    "    Observe=1000#The number of steps of observation before the beginning of the training\n",
    "    Store_Memory = deque()\n",
    "    Cost_History=[]\n",
    "    Memory_Size = 5000\n",
    "    Obstacle=[20,3,5,16,21,23,27,29,31,33,34,56,57,66,67,78,81,86]\n",
    "    Play_Done=None\n",
    "    def __init__(self,Learning_Rate=0.001,Gamma=0.9,Memory_Size=5000):\n",
    "        self.Learning_Rate=Learning_Rate\n",
    "        self.Gamma=Gamma\n",
    "        self.Memory_Size=Memory_Size\n",
    "        self.New_Network()\n",
    "        self.session=tf.InteractiveSession()\n",
    "        self.session.run(tf.initialize_all_variables())\n",
    "     #***************************************************************\n",
    "    def New_Network(self):\n",
    "         #Construct a neural network\n",
    "        self.State_Input=tf.placeholder(tf.float32,[None,self.State_Number])#Input_Number=100, M=Batch*100\n",
    "        self.Action_Input=tf.placeholder(tf.float32,[None,self.Action_Number])#Input_Number=4, M=Batch*4\n",
    "        self.Q_Target_OI=tf.placeholder(dtype=tf.float32,shape=[None])\n",
    "        #Layer1\n",
    "        Layer1_Number=20\n",
    "        w1=tf.Variable(tf.random_normal([self.State_Number,Layer1_Number]))\n",
    "        b1=tf.Variable(tf.zeros([1,Layer1_Number])+0.1)\n",
    "        l1=tf.nn.relu(tf.matmul(self.State_Input,w1)+b1) #b1 use the propagation mechanism\n",
    "#         #Layer2\n",
    "#         Layer2_Number=10\n",
    "#         w2=tf.Variable(tf.random_normal([Layer1_Number,Layer2_Number]))\n",
    "#         b2=tf.Variable(tf.zeros([1,Layer2_Number]))\n",
    "#         l2=tf.nn.relu(tf.matmul(l1,w2)+b2)\n",
    "        #Layer3\n",
    "        Layer3_Number=self.Action_Number\n",
    "        w3=tf.Variable(tf.random_normal([Layer1_Number,Layer3_Number]))\n",
    "        b3=tf.Variable(tf.zeros([1,Layer3_Number]))\n",
    "        self.l3=tf.matmul(l1,w3)+b3#At this time, not yet use the active action, l3 is the output matrice, M=Batch*4\n",
    "\n",
    "        self.Q_Value=tf.reduce_sum(tf.multiply(self.l3,self.Action_Input),reduction_indices=1)\n",
    "        self.Loss=tf.reduce_mean(tf.square(self.Q_Value-self.Q_Target_OI))\n",
    "        self.Optimizer=tf.train.GradientDescentOptimizer(self.Learning_Rate).minimize(self.Loss)\n",
    "        #Network's Output is like this form [19,20,297,30], argmax is 2, then choose Action2  \n",
    "        self.Predict=tf.argmax(self.l3,1)\n",
    "     #***************************************************************\n",
    "    def Select_Action(self,State_Index):\n",
    "        Current_State=self.State_List[State_Index:State_Index+1] #By example, Action_Index is 2, Current_State=[0,0,1,0， ，]\n",
    "        if np.random.uniform()<self.Epsilon:\n",
    "            Choose_Action_Index=np.random.randint(0,self.Action_Number)\n",
    "        else:\n",
    "            Action_QValue_Output=self.session.run(self.l3,feed_dict={self.State_Input:Current_State})\n",
    "            Choose_Action_Index=np.argmax(Action_QValue_Output)\n",
    "        #The first inegality means the train has beginned\n",
    "        #The second inegality means Epsilon has not yet decreased to final value\n",
    "        self.Epsilon=self.Epsilon_Final+(self.Epsilon_Begin-self.Epsilon_Final)*np.exp(-self.decay_rate*self.Step)\n",
    "#         if(self.Step%1000==0):\n",
    "#             print(\"epsilon \",self.Epsilon)\n",
    "        return Choose_Action_Index\n",
    "     #***************************************************************\n",
    "    def Next_State(self,Action_Index,State_Index):\n",
    "        State=State_Index+1\n",
    "        done=False\n",
    "        #Execute left motion\n",
    "        if(Action_Index==0):\n",
    "            if(State%self.N_Mesh==1):\n",
    "                State=State\n",
    "                R=-1000\n",
    "            elif((State+1) in self.Obstacle):#When encounter obstables, we stay \n",
    "                State=State\n",
    "                R=-1000\n",
    "            else:\n",
    "                State=State-1\n",
    "                R=0\n",
    "        elif(Action_Index==1):\n",
    "            if(State==self.State_Number-1):\n",
    "                State=State+1\n",
    "                done=True\n",
    "                R=100\n",
    "            elif(State%self.N_Mesh==0):\n",
    "                State=State\n",
    "                R=-1000\n",
    "            elif((State+1) in self.Obstacle):\n",
    "                State=State\n",
    "                R=-1000\n",
    "            else:\n",
    "                State=State+1\n",
    "                R=0.02\n",
    "        elif(Action_Index==2):\n",
    "            if(State==self.State_Number-self.N_Mesh):\n",
    "                State=State+self.N_Mesh\n",
    "                done=True\n",
    "                R=100\n",
    "            elif(self.State_Number+1-self.N_Mesh<=State<=self.State_Number-1):\n",
    "                State=State\n",
    "                R=-1000\n",
    "            elif((State+self.N_Mesh) in self.Obstacle):\n",
    "                State=State+self.N_Mesh\n",
    "                R=-1000\n",
    "            else:\n",
    "                State=State+self.N_Mesh\n",
    "                R=0.02\n",
    "        else:\n",
    "            if(0<=State<=self.N_Mesh):\n",
    "                State=State\n",
    "                R=-100\n",
    "            elif((State-self.N_Mesh) in self.Obstacle):\n",
    "                State=State-self.N_Mesh\n",
    "                R=-100\n",
    "            else:\n",
    "                R=0\n",
    "                State=State-self.N_Mesh\n",
    "        return State-1,R,done\n",
    "    \n",
    "    def Save_Memory(self,CURRENT_STATET,CHOOSE_ACTION,NEXT_STATE,REWARD,DONE):\n",
    "        Memory_Current_State=self.State_List[CURRENT_STATET:CURRENT_STATET+1]\n",
    "        Memory_Choose_Action=self.Action_List[CHOOSE_ACTION:CHOOSE_ACTION+1]\n",
    "        Memory_Next_State=self.State_List[NEXT_STATE:NEXT_STATE+1]\n",
    "        self.Store_Memory.append((Memory_Current_State,Memory_Choose_Action,Memory_Next_State,REWARD,DONE))\n",
    "        if len(self.Store_Memory)>self.Memory_Size:\n",
    "            self.Store_Memory.popleft()\n",
    "    \n",
    "     #***************************************************************\n",
    "    def Experience_Replay(self):\n",
    "        Batch=self.Batch_Number\n",
    "        MiniBatch=random.sample(self.Store_Memory,Batch)\n",
    "        Batch_Current_State = None\n",
    "        Batch_Execute_Action = None\n",
    "        Batch_Reward = None\n",
    "        Batch_Next_State = None\n",
    "        Batch_Done = None\n",
    "        \n",
    "        for Index in range(Batch):\n",
    "            if Batch_Current_State is None:\n",
    "                Batch_Current_State=MiniBatch[Index][0]\n",
    "            elif Batch_Current_State is not None:\n",
    "                Batch_Current_State=np.vstack((Batch_Current_State, MiniBatch[Index][0]))\n",
    "            #---------------------------------------------------------------------------------------   \n",
    "            #print(\"BEA\",MiniBatch[Index][1].shape)\n",
    "            if Batch_Execute_Action is None:\n",
    "                Batch_Execute_Action=MiniBatch[Index][1]\n",
    "            elif Batch_Execute_Action is not None:\n",
    "                Batch_Execute_Action=np.vstack((Batch_Execute_Action,MiniBatch[Index][1]))\n",
    "           #---------------------------------------------------------------------------------------           \n",
    "            if Batch_Reward is None:\n",
    "                Batch_Reward=MiniBatch[Index][3]\n",
    "            elif Batch_Reward is not None:\n",
    "                Batch_Reward=np.vstack((Batch_Reward,MiniBatch[Index][3]))\n",
    "            #---------------------------------------------------------------------------------------  \n",
    "            if Batch_Next_State is None:\n",
    "                Batch_Next_State=MiniBatch[Index][2]\n",
    "            elif Batch_Next_State is not None:\n",
    "                Batch_Next_State=np.vstack((Batch_Next_State,MiniBatch[Index][2]))\n",
    "            #---------------------------------------------------------------------------------------      \n",
    "            if Batch_Done is None:\n",
    "                Batch_Done=MiniBatch[Index][4]\n",
    "            elif Batch_Done is not None:\n",
    "                Batch_Done=np.vstack((Batch_Done,MiniBatch[Index][4]))\n",
    "            #Calculate the Q Value of the next State \n",
    "        Q_Next=self.session.run(self.l3,feed_dict={self.State_Input:Batch_Next_State})\n",
    "        Q_Target=[]\n",
    "        for i in range(Batch):\n",
    "            Each_Reward=Batch_Reward[i][0]#This is a 2D array because of the vstack\n",
    "            #Calculate the Target-Q-Value of each element in the Batch\n",
    "            Each_QValue=Each_Reward+self.Gamma*np.max(Q_Next[i]) #The network ouput has its own []\n",
    "            if Each_Reward<0:\n",
    "                Q_Target.append(Each_Reward)\n",
    "            else:\n",
    "                Q_Target.append(Each_QValue)\n",
    "        #print(self.session.run(self.Q_Value,feed_dict={self.Q_Target_OI:Q_Target}))\n",
    "        _,Cost,Rew=self.session.run([self.Q_Value,self.Loss,self.Optimizer],feed_dict={self.State_Input:Batch_Current_State,\n",
    "                                                                                        self.Action_Input:Batch_Execute_Action,\n",
    "                                                                                        self.Q_Target_OI: Q_Target})\n",
    "        self.Cost_History.append(Cost)\n",
    "        if self.Step%5000==0:\n",
    "            print(self.Step)  \n",
    "            print(\"Loss:\", Cost)  \n",
    "            print(\"Epsilon:\", self.Epsilon)   \n",
    "    #**************************************************************\n",
    "    def Train(self):\n",
    "        Train_Current_State=np.random.randint(0,self.State_Number-1)\n",
    "        self.Epsilon = self.Epsilon_Begin\n",
    "        while True:\n",
    "            Train_Action=self.Select_Action(Train_Current_State)\n",
    "            #print(\"TA\",Train_Action)\n",
    "            Train_Next_State,Train_Reward,Train_Done=self.Next_State(Train_Action,Train_Current_State)\n",
    "            self.Save_Memory(Train_Current_State,Train_Action,Train_Next_State,Train_Reward,Train_Done)\n",
    "            if self.Step>self.Observe:\n",
    "                self.Experience_Replay()\n",
    "            if self.Step>500000:\n",
    "                 break;\n",
    "            if Train_Done:\n",
    "                Train_Current_State=np.random.randint(0,self.State_Number-1)\n",
    "            else:\n",
    "                Train_Current_State=Train_Next_State\n",
    "                self.Step+=1          \n",
    "    #***************************************************************\n",
    "    def Play(self):\n",
    "        self.Train()\n",
    "        Start_Room_Index=0;\n",
    "        Play_Current_State_Index=0\n",
    "        Play_Step=0;\n",
    "        print(\"****************** Agent is in the State\",Start_Room_Index,\"**********************\")\n",
    "        while(Play_Current_State_Index!=self.State_Number-1):\n",
    "            Play_Current_State=self.State_List[Play_Current_State_Index:Play_Current_State_Index+1]\n",
    "            Play_Action=(self.session.run(self.Predict,feed_dict={self.State_Input:Play_Current_State}))\n",
    "            if Play_Action==0:\n",
    "                Play_Current_State_Index=Play_Current_State_Index-1\n",
    "            elif Play_Action==1:\n",
    "                Play_Current_State_Index=Play_Current_State_Index+1\n",
    "            elif Play_Action==2:\n",
    "                Play_Current_State_Index=Play_Current_State_Index+self.N_Mesh\n",
    "            else:\n",
    "                Play_Current_State_Index=Play_Current_State_Index-self.N_Mesh\n",
    "            Play_Step+=1\n",
    "            print(\"Step is \",Play_Step,\"****************** Agent is in the State\", Play_Current_State_Index,\"**********************\")\n",
    "            if(Play_Step>30):\n",
    "                break\n",
    "        if Play_Current_State_Index==self.State_Number-1:\n",
    "            self.Graph()\n",
    "    #***************************************************************        \n",
    "    def Caculate_coordinate(self,State):\n",
    "        X_Axis=0.5+(State-1)%self.N_Mesh\n",
    "        Y_Axis=0.5+((State-1)//self.N_Mesh)\n",
    "        return X_Axis,Y_Axis \n",
    "    #***************************************************************  \n",
    "    def Graph(self): \n",
    "        Graph_Step=0\n",
    "        fig=plt.figure()\n",
    "        ax=fig.gca()\n",
    "        ax.set(xlim=[0, self.N_Mesh], ylim=[0, self.N_Mesh])\n",
    "        ax.set_xticks(np.arange(0,(self.N_Mesh+1)))\n",
    "        ax.set_yticks(np.arange(0,(self.N_Mesh+1)))\n",
    "        plt.grid()\n",
    "        Graph_Start_Index=0\n",
    "        Graph_Current_Index=Graph_Start_Index\n",
    "        while(Graph_Current_Index!=self.State_Number-1):\n",
    "            Graph_Current_State=self.State_List[Graph_Current_Index:Graph_Current_Index+1]\n",
    "            Graph_Action=(self.session.run(self.Predict,feed_dict={self.State_Input:Graph_Current_State}))\n",
    "            if(Graph_Action==0):\n",
    "                Graph_Next_Index=Graph_Current_Index-1\n",
    "            elif(Graph_Action==1):\n",
    "                Graph_Next_Index=Graph_Current_Index+1\n",
    "            elif(Graph_Action==2):\n",
    "                 Graph_Next_Index=Graph_Current_Index+self.N_Mesh\n",
    "            else:\n",
    "                 Graph_Next_Index=Graph_Current_Index-self.N_Mesh\n",
    "            Graph_Step+=1\n",
    "            X_State,Y_State=self.Caculate_coordinate(Graph_Current_Index+1)\n",
    "            X_Next_State,Y_Next_State=self.Caculate_coordinate(Graph_Next_Index+1)\n",
    "            if(Graph_Step==0):\n",
    "                plt.scatter(X_State,Y_State)  \n",
    "            else:\n",
    "                plt.scatter(X_State,Y_State) \n",
    "                plt.scatter(X_Next_State,Y_Next_State)\n",
    "                plt.plot([X_State,X_Next_State],[Y_State,Y_Next_State])\n",
    "            for J in np.arange(len(self.Obstacle)):\n",
    "                plt.scatter((self.Caculate_coordinate(self.Obstacle[J]+1))[0],(self.Caculate_coordinate(self.Obstacle[J]+1))[1],marker=\"x\")\n",
    "            Graph_Current_Index=Graph_Next_Index\n",
    "    plt.show()\n",
    "    #***************************************************************  \n",
    "    def Plot_loss(self): \n",
    "        plt.plot(self.Cost_History)\n",
    "   #***************************************************************          \n",
    "    def Draw_Obstacle(self):\n",
    "        print('Draw')\n",
    "        fig=plt.figure()\n",
    "        ax=fig.gca()\n",
    "        ax.set(xlim=[0, self.N_Mesh], ylim=[0, self.N_Mesh])\n",
    "        ax.set_xticks(np.arange(0,(self.N_Mesh+1)))\n",
    "        ax.set_yticks(np.arange(0,(self.N_Mesh+1)))\n",
    "        plt.grid()\n",
    "        for J in np.arange(len(self.Obstacle)):\n",
    "                plt.scatter((self.Caculate_coordinate(self.Obstacle[J]+1))[0],(self.Caculate_coordinate(self.Obstacle[J]+1))[1],marker=\"x\")\n",
    "        plt.show()\n",
    "if __name__ == \"__main__\":\n",
    "    q_network = Q_Network()\n",
    "    q_network.Play()\n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib qt5\n",
    "A=q_network.Cost_History[0:3000]\n",
    "np.savetxt('Loss1.csv',A)\n",
    "# plt.plot(A)\n",
    "# plt.xlabel('time-steps',size=30)\n",
    "# plt.ylabel('loss value',size=30)\n",
    "# plt.tick_params(labelsize=30)\n",
    "# plt.title('Loss',fontsize=40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "q_network.Plot_loss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
